#!/usr/bin/env python
# -*- coding: UTF-8 -*-
'''
@Project ：图灵学习 
@File    ：1.单线程爬虫-数据去重.py
@IDE     ：PyCharm 
@Author  ：ChenLiGao
@Date    ：2024-12-04 10:21
'''
import redis
import requests
import pymongo
import hashlib # 将夺取的数据进行hash，md5的格式数据


class MovieInfo:
    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
    }

    api_url = "https://pianku.api.mgtv.com/rider/list/pcweb/v3"

    def __init__(self):
        self.client_mongo = pymongo.MongoClient(host="localhost", port=27017)
        self.db_mongo = self.client_mongo["py_spider"]['mg_movie_info']
        self.db_redis = redis.Redis(host="localhost", port=6379, db=0)

    def __del__(self):
        self.db_redis.close()


    # 数据请求
    @classmethod
    def get_movie_info(cls, params_attr):
        response = requests.get(cls.api_url, headers=cls.headers, params=params_attr)
        return response.json()

    # 数据提取并调整数据结构
    def parse_movide_info(self, response_data):
        movie_list = response_data['data']['hitDocs']

        for movie in movie_list:
            item = dict()
            item["title"] = movie["title"]
            item["subtitle"]= movie['subtitle']
            item["story"] = movie['story']

            # 当数据结构调整完毕后进行数据保存
            self.save_movie_info(item)

    # 完成数据去重
    @staticmethod
    def get_md5(value):
        md5_hash = hashlib.md5(str(value).encode("utf-8")).hexdigest()
        return md5_hash


    # 保存数据方法
    def save_movie_info(self, item):
        md5_hash = self.get_md5(item)
        result = self.db_redis.sadd("moive:filter", md5_hash)
        if result:
            self.db_mongo.insert_one(item)
            print("插入数据成功：", item)
        else:
            print("数据重复，不插入")


    def main(self):
        for page in range(1, 3):
            params = {
                "allowedRC": "1",
                "platform": "pcweb",
                "channelId": "2",
                "pn": page,
                "pc": "80",
                "hudong": "1",
                "_support": "10000000",
                "kind": "19",
                "area": "10",
                "year": "all",
                "chargeInfo": "a1",
                "sort": "c2",
                "feature": "all"
            }

            response_data = self.get_movie_info(params)
            self.parse_movide_info(response_data)


if __name__ == '__main__':
    movie_info = MovieInfo()
    movie_info.main()
